#ifndef __PAGE_LIB_PREPROCESSOR_HH__
#define __PAGE_LIB_PREPROCESSOR_HH__


#include "WebPage.hh"
#include "../SplitToolCppJieba.hh"
#include "../SplitTool.hh"
#include "../Configuration.hh"
#include "../simhash/Simhasher.hpp"
#include "tinyxml2.h"
#include <vector>
#include <unordered_map>

using namespace simhash;
using std::vector;
using std::unordered_map;
using std::pair;

class PageLibPreprocessor
{
public:
    PageLibPreprocessor(SplitTool* tool);
    void doProcess();
    void readInfoFromFile();
    void cutRedundantPages();
    void bulidInvertIndexTable();
    void storeOnDisk();
private:
    void readRipePage();
    void readOffSet();
    void createOldFingerprint(Simhasher& simhasher, vector<pair<size_t, uint64_t>>& fingerprint);
    vector<pair<int, uint64_t>> createNewFingerprint(Simhasher& simhasher, vector<pair<size_t, uint64_t>>& fingerprint);
    void creatNewIndex(vector<pair<int, uint64_t>>& new_fingerprint, map<int, pair<size_t, size_t>>& new_index);
    void createNewRipePage(map<int, pair<size_t, size_t>>& new_index);


    void getWordAndPageId(map<string, set<int>>& ms);
    void getTF(map<string, set<int>>& ms);
    
private:
    SplitTool* _tool;
    vector<WebPage> _pageLib;
    unordered_map<int, pair<size_t, size_t>> _offsetLib; // 文章的编号，文章的位置
    
    // 倒排表, key 为单词，value 为文章编号和权重
    unordered_map<string, vector<pair<int, double>>> _invertIndexTable;

};

#endif